{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "#\"\"\"Bottleneck layers. Although each layer only produces k\n",
    "#output feature-maps, it typically has many more inputs. It\n",
    "#has been noted in [37, 11] that a 1×1 convolution can be in-\n",
    "#troduced as bottleneck layer before each 3×3 convolution\n",
    "#to reduce the number of input feature-maps, and thus to\n",
    "#improve computational efficiency.\"\"\"\n",
    "class Bottleneck(nn.Module):\n",
    "    def __init__(self, in_channels, growth_rate, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        #\"\"\"In  our experiments, we let each 1×1 convolution \n",
    "        #produce 4k feature-maps.\"\"\"\n",
    "        inner_channel = 4 * growth_rate\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "\n",
    "        #\"\"\"We find this design especially effective for DenseNet and \n",
    "        #we refer to our network with such a bottleneck layer, i.e., \n",
    "        #to the BN-ReLU-Conv(1×1)-BN-ReLU-Conv(3×3) version of H ` , \n",
    "        #as DenseNet-B.\"\"\"\n",
    "        self.bottle_neck = nn.Sequential(\n",
    "            nn.BatchNorm2d(in_channels),\n",
    "            f_activation,\n",
    "            nn.Conv2d(in_channels, inner_channel, kernel_size=1, bias=False),\n",
    "            nn.BatchNorm2d(inner_channel),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(inner_channel, growth_rate, kernel_size=3, padding=1, bias=False)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return torch.cat([x, self.bottle_neck(x)], 1)\n",
    "\n",
    "#\"\"\"We refer to layers between blocks as transition\n",
    "#layers, which do convolution and pooling.\"\"\"\n",
    "class Transition(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels):\n",
    "        super().__init__()\n",
    "        #\"\"\"The transition layers used in our experiments \n",
    "        #consist of a batch normalization layer and an 1×1 \n",
    "        #convolutional layer followed by a 2×2 average pooling \n",
    "        #layer\"\"\".\n",
    "        self.down_sample = nn.Sequential(\n",
    "            nn.BatchNorm2d(in_channels),\n",
    "            nn.Conv2d(in_channels, out_channels, 1, bias=False),\n",
    "            nn.AvgPool2d(2, stride=2)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.down_sample(x)\n",
    "\n",
    "#DesneNet-BC\n",
    "#B stands for bottleneck layer(BN-RELU-CONV(1x1)-BN-RELU-CONV(3x3))\n",
    "#C stands for compression factor(0<=theta<=1)\n",
    "class DenseNet(nn.Module):\n",
    "    def __init__(self, block, nblocks, growth_rate=12, reduction=0.5, num_class=100, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        self.growth_rate = growth_rate\n",
    "\n",
    "        #\"\"\"Before entering the first dense block, a convolution \n",
    "        #with 16 (or twice the growth rate for DenseNet-BC) \n",
    "        #output channels is performed on the input images.\"\"\"\n",
    "        inner_channels = 2 * growth_rate\n",
    "\n",
    "        #For convolutional layers with kernel size 3×3, each \n",
    "        #side of the inputs is zero-padded by one pixel to keep \n",
    "        #the feature-map size fixed.\n",
    "        self.conv1 = nn.Conv2d(3, inner_channels, kernel_size=3, padding=1, bias=False) \n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "\n",
    "        self.features = nn.Sequential()\n",
    "\n",
    "        for index in range(len(nblocks) - 1):\n",
    "            self.features.add_module(\"dense_block_layer_{}\".format(index), self._make_dense_layers(block, inner_channels, nblocks[index]))\n",
    "            inner_channels += growth_rate * nblocks[index]\n",
    "\n",
    "            #\"\"\"If a dense block contains m feature-maps, we let the \n",
    "            #following transition layer generate θm output feature-\n",
    "            #maps, where 0 < θ ≤ 1 is referred to as the compression \n",
    "            #fac-tor.\n",
    "            out_channels = int(reduction * inner_channels) # int() will automatic floor the value\n",
    "            self.features.add_module(\"transition_layer_{}\".format(index), Transition(inner_channels, out_channels))\n",
    "            inner_channels = out_channels\n",
    "\n",
    "        self.features.add_module(\"dense_block{}\".format(len(nblocks) - 1), self._make_dense_layers(block, inner_channels, nblocks[len(nblocks)-1]))\n",
    "        inner_channels += growth_rate * nblocks[len(nblocks) - 1]\n",
    "        self.features.add_module('bn', nn.BatchNorm2d(inner_channels))\n",
    "        self.features.add_module('activation', f_activation)\n",
    "\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n",
    "\n",
    "        self.linear = nn.Linear(inner_channels, num_class)\n",
    "\n",
    "    def forward(self, x):\n",
    "        output = self.conv1(x)\n",
    "        output = self.features(output)\n",
    "        output = self.avgpool(output)\n",
    "        output = output.view(output.size()[0], -1)\n",
    "        output = self.linear(output)\n",
    "        return output\n",
    "\n",
    "    def _make_dense_layers(self, block, in_channels, nblocks):\n",
    "        dense_block = nn.Sequential()\n",
    "        for index in range(nblocks):\n",
    "            dense_block.add_module('bottle_neck_layer_{}'.format(index), block(in_channels, self.growth_rate))\n",
    "            in_channels += self.growth_rate\n",
    "        return dense_block\n",
    "\n",
    "def densenet121(activation = 'relu'):\n",
    "    return DenseNet(Bottleneck, [6,12,24,16], growth_rate=32, activation = activation)\n",
    "\n",
    "def densenet169(activation = 'relu'):\n",
    "    return DenseNet(Bottleneck, [6,12,32,32], growth_rate=32, activation = activation)\n",
    "\n",
    "def densenet201(activation = 'relu'):\n",
    "    return DenseNet(Bottleneck, [6,12,48,32], growth_rate=32, activation = activation)\n",
    "\n",
    "def densenet161(activation = 'relu'):\n",
    "    return DenseNet(Bottleneck, [6,12,36,24], growth_rate=48, activation = activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:06, 27250770.14it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = densenet169(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 139.7525.. Average time per step: 0.3574.. Train loss: 1.5018.. Train accuracy: 0.4577.. Top-3 train accuracy: 0.7917.. Test loss: 1.2931.. Test accuracy: 0.5429.. Top-3 test accuracy: 0.8339\n",
      "Epoch 2/100.. Time per epoch: 139.3649.. Average time per step: 0.3564.. Train loss: 1.0871.. Train accuracy: 0.6102.. Top-3 train accuracy: 0.8833.. Test loss: 0.9734.. Test accuracy: 0.6484.. Top-3 test accuracy: 0.9106\n",
      "Epoch 3/100.. Time per epoch: 139.6079.. Average time per step: 0.3571.. Train loss: 0.9033.. Train accuracy: 0.6789.. Top-3 train accuracy: 0.9127.. Test loss: 0.8966.. Test accuracy: 0.6880.. Top-3 test accuracy: 0.9142\n",
      "Epoch 4/100.. Time per epoch: 139.3116.. Average time per step: 0.3563.. Train loss: 0.7871.. Train accuracy: 0.7217.. Top-3 train accuracy: 0.9327.. Test loss: 0.7237.. Test accuracy: 0.7507.. Top-3 test accuracy: 0.9418\n",
      "Epoch 5/100.. Time per epoch: 139.2973.. Average time per step: 0.3563.. Train loss: 0.6856.. Train accuracy: 0.7607.. Top-3 train accuracy: 0.9474.. Test loss: 0.6018.. Test accuracy: 0.7935.. Top-3 test accuracy: 0.9589\n",
      "Epoch 6/100.. Time per epoch: 139.3799.. Average time per step: 0.3565.. Train loss: 0.6138.. Train accuracy: 0.7850.. Top-3 train accuracy: 0.9564.. Test loss: 0.5628.. Test accuracy: 0.8062.. Top-3 test accuracy: 0.9616\n",
      "Epoch 7/100.. Time per epoch: 139.2817.. Average time per step: 0.3562.. Train loss: 0.5477.. Train accuracy: 0.8086.. Top-3 train accuracy: 0.9626.. Test loss: 0.5250.. Test accuracy: 0.8200.. Top-3 test accuracy: 0.9670\n",
      "Epoch 8/100.. Time per epoch: 139.3864.. Average time per step: 0.3565.. Train loss: 0.5099.. Train accuracy: 0.8229.. Top-3 train accuracy: 0.9666.. Test loss: 0.4803.. Test accuracy: 0.8351.. Top-3 test accuracy: 0.9700\n",
      "Epoch 9/100.. Time per epoch: 139.3447.. Average time per step: 0.3564.. Train loss: 0.4655.. Train accuracy: 0.8382.. Top-3 train accuracy: 0.9726.. Test loss: 0.4354.. Test accuracy: 0.8497.. Top-3 test accuracy: 0.9761\n",
      "Epoch 10/100.. Time per epoch: 139.3978.. Average time per step: 0.3565.. Train loss: 0.4306.. Train accuracy: 0.8499.. Top-3 train accuracy: 0.9743.. Test loss: 0.4402.. Test accuracy: 0.8534.. Top-3 test accuracy: 0.9727\n",
      "Epoch 11/100.. Time per epoch: 139.0216.. Average time per step: 0.3556.. Train loss: 0.4022.. Train accuracy: 0.8581.. Top-3 train accuracy: 0.9781.. Test loss: 0.3867.. Test accuracy: 0.8659.. Top-3 test accuracy: 0.9779\n",
      "Epoch 12/100.. Time per epoch: 139.1519.. Average time per step: 0.3559.. Train loss: 0.3728.. Train accuracy: 0.8701.. Top-3 train accuracy: 0.9802.. Test loss: 0.3792.. Test accuracy: 0.8672.. Top-3 test accuracy: 0.9803\n",
      "Epoch 13/100.. Time per epoch: 139.3862.. Average time per step: 0.3565.. Train loss: 0.3503.. Train accuracy: 0.8761.. Top-3 train accuracy: 0.9819.. Test loss: 0.4433.. Test accuracy: 0.8508.. Top-3 test accuracy: 0.9771\n",
      "Epoch 14/100.. Time per epoch: 139.3114.. Average time per step: 0.3563.. Train loss: 0.3211.. Train accuracy: 0.8873.. Top-3 train accuracy: 0.9848.. Test loss: 0.3722.. Test accuracy: 0.8737.. Top-3 test accuracy: 0.9815\n",
      "Epoch 15/100.. Time per epoch: 139.2456.. Average time per step: 0.3561.. Train loss: 0.3092.. Train accuracy: 0.8914.. Top-3 train accuracy: 0.9862.. Test loss: 0.3725.. Test accuracy: 0.8783.. Top-3 test accuracy: 0.9807\n",
      "Epoch 16/100.. Time per epoch: 139.2121.. Average time per step: 0.3560.. Train loss: 0.2826.. Train accuracy: 0.8991.. Top-3 train accuracy: 0.9880.. Test loss: 0.3702.. Test accuracy: 0.8770.. Top-3 test accuracy: 0.9819\n",
      "Epoch 17/100.. Time per epoch: 139.2943.. Average time per step: 0.3563.. Train loss: 0.2691.. Train accuracy: 0.9060.. Top-3 train accuracy: 0.9890.. Test loss: 0.3669.. Test accuracy: 0.8798.. Top-3 test accuracy: 0.9834\n",
      "Epoch 18/100.. Time per epoch: 139.0387.. Average time per step: 0.3556.. Train loss: 0.2480.. Train accuracy: 0.9131.. Top-3 train accuracy: 0.9902.. Test loss: 0.3406.. Test accuracy: 0.8897.. Top-3 test accuracy: 0.9836\n",
      "Epoch 19/100.. Time per epoch: 139.3085.. Average time per step: 0.3563.. Train loss: 0.2315.. Train accuracy: 0.9184.. Top-3 train accuracy: 0.9912.. Test loss: 0.3628.. Test accuracy: 0.8859.. Top-3 test accuracy: 0.9843\n",
      "Epoch 20/100.. Time per epoch: 139.2053.. Average time per step: 0.3560.. Train loss: 0.2174.. Train accuracy: 0.9226.. Top-3 train accuracy: 0.9929.. Test loss: 0.3322.. Test accuracy: 0.8939.. Top-3 test accuracy: 0.9847\n",
      "Epoch 21/100.. Time per epoch: 139.2424.. Average time per step: 0.3561.. Train loss: 0.2037.. Train accuracy: 0.9283.. Top-3 train accuracy: 0.9934.. Test loss: 0.3374.. Test accuracy: 0.8948.. Top-3 test accuracy: 0.9850\n",
      "Epoch 22/100.. Time per epoch: 138.8590.. Average time per step: 0.3551.. Train loss: 0.1905.. Train accuracy: 0.9322.. Top-3 train accuracy: 0.9938.. Test loss: 0.3772.. Test accuracy: 0.8836.. Top-3 test accuracy: 0.9825\n",
      "Epoch 23/100.. Time per epoch: 139.1772.. Average time per step: 0.3560.. Train loss: 0.1810.. Train accuracy: 0.9360.. Top-3 train accuracy: 0.9948.. Test loss: 0.3476.. Test accuracy: 0.8920.. Top-3 test accuracy: 0.9837\n",
      "Epoch 24/100.. Time per epoch: 139.1825.. Average time per step: 0.3560.. Train loss: 0.1657.. Train accuracy: 0.9418.. Top-3 train accuracy: 0.9956.. Test loss: 0.3520.. Test accuracy: 0.8902.. Top-3 test accuracy: 0.9852\n",
      "Epoch 25/100.. Time per epoch: 139.4047.. Average time per step: 0.3565.. Train loss: 0.1605.. Train accuracy: 0.9438.. Top-3 train accuracy: 0.9954.. Test loss: 0.3489.. Test accuracy: 0.8985.. Top-3 test accuracy: 0.9838\n",
      "Epoch 26/100.. Time per epoch: 139.1113.. Average time per step: 0.3558.. Train loss: 0.1488.. Train accuracy: 0.9465.. Top-3 train accuracy: 0.9965.. Test loss: 0.3562.. Test accuracy: 0.8985.. Top-3 test accuracy: 0.9849\n",
      "Epoch 27/100.. Time per epoch: 139.1071.. Average time per step: 0.3558.. Train loss: 0.1421.. Train accuracy: 0.9492.. Top-3 train accuracy: 0.9965.. Test loss: 0.3433.. Test accuracy: 0.8993.. Top-3 test accuracy: 0.9847\n",
      "Epoch 28/100.. Time per epoch: 139.6162.. Average time per step: 0.3571.. Train loss: 0.1321.. Train accuracy: 0.9536.. Top-3 train accuracy: 0.9970.. Test loss: 0.3625.. Test accuracy: 0.8986.. Top-3 test accuracy: 0.9869\n",
      "Epoch 29/100.. Time per epoch: 139.6668.. Average time per step: 0.3572.. Train loss: 0.1280.. Train accuracy: 0.9539.. Top-3 train accuracy: 0.9971.. Test loss: 0.3577.. Test accuracy: 0.8969.. Top-3 test accuracy: 0.9864\n",
      "Epoch 30/100.. Time per epoch: 139.3999.. Average time per step: 0.3565.. Train loss: 0.1194.. Train accuracy: 0.9578.. Top-3 train accuracy: 0.9978.. Test loss: 0.3562.. Test accuracy: 0.8995.. Top-3 test accuracy: 0.9861\n",
      "Epoch 31/100.. Time per epoch: 139.1888.. Average time per step: 0.3560.. Train loss: 0.1107.. Train accuracy: 0.9607.. Top-3 train accuracy: 0.9979.. Test loss: 0.3536.. Test accuracy: 0.9058.. Top-3 test accuracy: 0.9857\n",
      "Epoch 32/100.. Time per epoch: 139.0797.. Average time per step: 0.3557.. Train loss: 0.1063.. Train accuracy: 0.9632.. Top-3 train accuracy: 0.9980.. Test loss: 0.3870.. Test accuracy: 0.8956.. Top-3 test accuracy: 0.9857\n",
      "Epoch 33/100.. Time per epoch: 139.0788.. Average time per step: 0.3557.. Train loss: 0.1026.. Train accuracy: 0.9636.. Top-3 train accuracy: 0.9981.. Test loss: 0.3658.. Test accuracy: 0.9039.. Top-3 test accuracy: 0.9848\n",
      "Epoch 34/100.. Time per epoch: 138.9980.. Average time per step: 0.3555.. Train loss: 0.1011.. Train accuracy: 0.9640.. Top-3 train accuracy: 0.9985.. Test loss: 0.3970.. Test accuracy: 0.8985.. Top-3 test accuracy: 0.9867\n",
      "Epoch 35/100.. Time per epoch: 138.9209.. Average time per step: 0.3553.. Train loss: 0.0941.. Train accuracy: 0.9658.. Top-3 train accuracy: 0.9985.. Test loss: 0.3995.. Test accuracy: 0.8976.. Top-3 test accuracy: 0.9852\n",
      "Epoch 36/100.. Time per epoch: 139.1545.. Average time per step: 0.3559.. Train loss: 0.0864.. Train accuracy: 0.9693.. Top-3 train accuracy: 0.9987.. Test loss: 0.3716.. Test accuracy: 0.9039.. Top-3 test accuracy: 0.9864\n",
      "Epoch 37/100.. Time per epoch: 139.3005.. Average time per step: 0.3563.. Train loss: 0.0831.. Train accuracy: 0.9707.. Top-3 train accuracy: 0.9989.. Test loss: 0.3718.. Test accuracy: 0.9035.. Top-3 test accuracy: 0.9864\n",
      "Epoch 38/100.. Time per epoch: 139.1893.. Average time per step: 0.3560.. Train loss: 0.0871.. Train accuracy: 0.9697.. Top-3 train accuracy: 0.9986.. Test loss: 0.3709.. Test accuracy: 0.9037.. Top-3 test accuracy: 0.9873\n",
      "Epoch 39/100.. Time per epoch: 139.2326.. Average time per step: 0.3561.. Train loss: 0.0787.. Train accuracy: 0.9717.. Top-3 train accuracy: 0.9991.. Test loss: 0.3880.. Test accuracy: 0.9035.. Top-3 test accuracy: 0.9842\n",
      "Epoch 40/100.. Time per epoch: 139.3263.. Average time per step: 0.3563.. Train loss: 0.0753.. Train accuracy: 0.9734.. Top-3 train accuracy: 0.9990.. Test loss: 0.4030.. Test accuracy: 0.9016.. Top-3 test accuracy: 0.9855\n",
      "Epoch 41/100.. Time per epoch: 139.1729.. Average time per step: 0.3559.. Train loss: 0.0712.. Train accuracy: 0.9745.. Top-3 train accuracy: 0.9990.. Test loss: 0.4004.. Test accuracy: 0.9009.. Top-3 test accuracy: 0.9873\n",
      "Epoch 42/100.. Time per epoch: 139.0736.. Average time per step: 0.3557.. Train loss: 0.0720.. Train accuracy: 0.9743.. Top-3 train accuracy: 0.9990.. Test loss: 0.4115.. Test accuracy: 0.8987.. Top-3 test accuracy: 0.9873\n",
      "Epoch 43/100.. Time per epoch: 139.1222.. Average time per step: 0.3558.. Train loss: 0.0729.. Train accuracy: 0.9740.. Top-3 train accuracy: 0.9991.. Test loss: 0.3909.. Test accuracy: 0.9026.. Top-3 test accuracy: 0.9864\n",
      "Epoch 44/100.. Time per epoch: 139.0871.. Average time per step: 0.3557.. Train loss: 0.0635.. Train accuracy: 0.9767.. Top-3 train accuracy: 0.9992.. Test loss: 0.4018.. Test accuracy: 0.9066.. Top-3 test accuracy: 0.9872\n",
      "Epoch 45/100.. Time per epoch: 139.0080.. Average time per step: 0.3555.. Train loss: 0.0605.. Train accuracy: 0.9790.. Top-3 train accuracy: 0.9992.. Test loss: 0.4088.. Test accuracy: 0.9037.. Top-3 test accuracy: 0.9866\n",
      "Epoch 46/100.. Time per epoch: 138.9833.. Average time per step: 0.3555.. Train loss: 0.0634.. Train accuracy: 0.9781.. Top-3 train accuracy: 0.9993.. Test loss: 0.4267.. Test accuracy: 0.9025.. Top-3 test accuracy: 0.9865\n",
      "Epoch 47/100.. Time per epoch: 138.8448.. Average time per step: 0.3551.. Train loss: 0.0604.. Train accuracy: 0.9788.. Top-3 train accuracy: 0.9994.. Test loss: 0.4286.. Test accuracy: 0.9001.. Top-3 test accuracy: 0.9844\n",
      "Epoch 48/100.. Time per epoch: 139.1544.. Average time per step: 0.3559.. Train loss: 0.0613.. Train accuracy: 0.9784.. Top-3 train accuracy: 0.9993.. Test loss: 0.4170.. Test accuracy: 0.9046.. Top-3 test accuracy: 0.9866\n",
      "Epoch 49/100.. Time per epoch: 139.0897.. Average time per step: 0.3557.. Train loss: 0.0566.. Train accuracy: 0.9796.. Top-3 train accuracy: 0.9995.. Test loss: 0.4077.. Test accuracy: 0.9046.. Top-3 test accuracy: 0.9856\n",
      "Epoch 50/100.. Time per epoch: 138.9526.. Average time per step: 0.3554.. Train loss: 0.0537.. Train accuracy: 0.9813.. Top-3 train accuracy: 0.9995.. Test loss: 0.4043.. Test accuracy: 0.9051.. Top-3 test accuracy: 0.9876\n",
      "Epoch 51/100.. Time per epoch: 138.8164.. Average time per step: 0.3550.. Train loss: 0.0543.. Train accuracy: 0.9815.. Top-3 train accuracy: 0.9994.. Test loss: 0.4534.. Test accuracy: 0.8959.. Top-3 test accuracy: 0.9840\n",
      "Epoch 52/100.. Time per epoch: 138.9421.. Average time per step: 0.3554.. Train loss: 0.0552.. Train accuracy: 0.9807.. Top-3 train accuracy: 0.9994.. Test loss: 0.4236.. Test accuracy: 0.9022.. Top-3 test accuracy: 0.9853\n",
      "Epoch 53/100.. Time per epoch: 138.8429.. Average time per step: 0.3551.. Train loss: 0.0544.. Train accuracy: 0.9810.. Top-3 train accuracy: 0.9995.. Test loss: 0.4399.. Test accuracy: 0.9035.. Top-3 test accuracy: 0.9850\n",
      "Epoch 54/100.. Time per epoch: 138.9521.. Average time per step: 0.3554.. Train loss: 0.0500.. Train accuracy: 0.9823.. Top-3 train accuracy: 0.9995.. Test loss: 0.4070.. Test accuracy: 0.9065.. Top-3 test accuracy: 0.9856\n",
      "Epoch 55/100.. Time per epoch: 138.7062.. Average time per step: 0.3547.. Train loss: 0.0459.. Train accuracy: 0.9841.. Top-3 train accuracy: 0.9994.. Test loss: 0.4544.. Test accuracy: 0.9042.. Top-3 test accuracy: 0.9851\n",
      "Epoch 56/100.. Time per epoch: 138.8707.. Average time per step: 0.3552.. Train loss: 0.0508.. Train accuracy: 0.9818.. Top-3 train accuracy: 0.9995.. Test loss: 0.4404.. Test accuracy: 0.9021.. Top-3 test accuracy: 0.9845\n",
      "Epoch 57/100.. Time per epoch: 138.8183.. Average time per step: 0.3550.. Train loss: 0.0503.. Train accuracy: 0.9816.. Top-3 train accuracy: 0.9994.. Test loss: 0.4454.. Test accuracy: 0.8992.. Top-3 test accuracy: 0.9844\n",
      "Epoch 58/100.. Time per epoch: 139.0571.. Average time per step: 0.3556.. Train loss: 0.0447.. Train accuracy: 0.9844.. Top-3 train accuracy: 0.9996.. Test loss: 0.4019.. Test accuracy: 0.9085.. Top-3 test accuracy: 0.9860\n",
      "Epoch 59/100.. Time per epoch: 139.6134.. Average time per step: 0.3571.. Train loss: 0.0429.. Train accuracy: 0.9845.. Top-3 train accuracy: 0.9997.. Test loss: 0.4246.. Test accuracy: 0.9063.. Top-3 test accuracy: 0.9869\n",
      "Epoch 60/100.. Time per epoch: 139.5506.. Average time per step: 0.3569.. Train loss: 0.0457.. Train accuracy: 0.9839.. Top-3 train accuracy: 0.9996.. Test loss: 0.4412.. Test accuracy: 0.9026.. Top-3 test accuracy: 0.9841\n",
      "Epoch 61/100.. Time per epoch: 139.5628.. Average time per step: 0.3569.. Train loss: 0.0407.. Train accuracy: 0.9856.. Top-3 train accuracy: 0.9996.. Test loss: 0.4489.. Test accuracy: 0.9033.. Top-3 test accuracy: 0.9866\n",
      "Epoch 62/100.. Time per epoch: 139.4049.. Average time per step: 0.3565.. Train loss: 0.0473.. Train accuracy: 0.9833.. Top-3 train accuracy: 0.9997.. Test loss: 0.4155.. Test accuracy: 0.9012.. Top-3 test accuracy: 0.9866\n",
      "Epoch 63/100.. Time per epoch: 138.9935.. Average time per step: 0.3555.. Train loss: 0.0404.. Train accuracy: 0.9863.. Top-3 train accuracy: 0.9998.. Test loss: 0.4477.. Test accuracy: 0.8986.. Top-3 test accuracy: 0.9856\n",
      "Epoch 64/100.. Time per epoch: 139.0345.. Average time per step: 0.3556.. Train loss: 0.0432.. Train accuracy: 0.9845.. Top-3 train accuracy: 0.9998.. Test loss: 0.4234.. Test accuracy: 0.9051.. Top-3 test accuracy: 0.9886\n",
      "Epoch 65/100.. Time per epoch: 139.2587.. Average time per step: 0.3562.. Train loss: 0.0406.. Train accuracy: 0.9859.. Top-3 train accuracy: 0.9996.. Test loss: 0.4497.. Test accuracy: 0.9033.. Top-3 test accuracy: 0.9864\n",
      "Epoch 66/100.. Time per epoch: 139.0498.. Average time per step: 0.3556.. Train loss: 0.0388.. Train accuracy: 0.9862.. Top-3 train accuracy: 0.9997.. Test loss: 0.4738.. Test accuracy: 0.9020.. Top-3 test accuracy: 0.9863\n",
      "Epoch 67/100.. Time per epoch: 138.9987.. Average time per step: 0.3555.. Train loss: 0.0357.. Train accuracy: 0.9873.. Top-3 train accuracy: 0.9997.. Test loss: 0.4459.. Test accuracy: 0.9050.. Top-3 test accuracy: 0.9872\n",
      "Epoch 68/100.. Time per epoch: 139.0706.. Average time per step: 0.3557.. Train loss: 0.0339.. Train accuracy: 0.9882.. Top-3 train accuracy: 0.9997.. Test loss: 0.4429.. Test accuracy: 0.9059.. Top-3 test accuracy: 0.9852\n",
      "Epoch 69/100.. Time per epoch: 139.3248.. Average time per step: 0.3563.. Train loss: 0.0404.. Train accuracy: 0.9855.. Top-3 train accuracy: 0.9997.. Test loss: 0.4586.. Test accuracy: 0.9010.. Top-3 test accuracy: 0.9859\n",
      "Epoch 70/100.. Time per epoch: 139.0030.. Average time per step: 0.3555.. Train loss: 0.0374.. Train accuracy: 0.9870.. Top-3 train accuracy: 0.9998.. Test loss: 0.4355.. Test accuracy: 0.9098.. Top-3 test accuracy: 0.9857\n",
      "Epoch 71/100.. Time per epoch: 139.2605.. Average time per step: 0.3562.. Train loss: 0.0397.. Train accuracy: 0.9869.. Top-3 train accuracy: 0.9998.. Test loss: 0.4445.. Test accuracy: 0.9086.. Top-3 test accuracy: 0.9857\n",
      "Epoch 72/100.. Time per epoch: 139.1406.. Average time per step: 0.3559.. Train loss: 0.0300.. Train accuracy: 0.9899.. Top-3 train accuracy: 0.9997.. Test loss: 0.4727.. Test accuracy: 0.9033.. Top-3 test accuracy: 0.9878\n",
      "Epoch 73/100.. Time per epoch: 139.1690.. Average time per step: 0.3559.. Train loss: 0.0349.. Train accuracy: 0.9878.. Top-3 train accuracy: 0.9998.. Test loss: 0.4661.. Test accuracy: 0.9033.. Top-3 test accuracy: 0.9869\n",
      "Epoch 74/100.. Time per epoch: 139.0891.. Average time per step: 0.3557.. Train loss: 0.0383.. Train accuracy: 0.9869.. Top-3 train accuracy: 0.9997.. Test loss: 0.4912.. Test accuracy: 0.9001.. Top-3 test accuracy: 0.9855\n",
      "Epoch 75/100.. Time per epoch: 139.0672.. Average time per step: 0.3557.. Train loss: 0.0330.. Train accuracy: 0.9882.. Top-3 train accuracy: 0.9998.. Test loss: 0.4626.. Test accuracy: 0.9037.. Top-3 test accuracy: 0.9872\n",
      "Epoch 76/100.. Time per epoch: 139.1627.. Average time per step: 0.3559.. Train loss: 0.0296.. Train accuracy: 0.9895.. Top-3 train accuracy: 0.9998.. Test loss: 0.4896.. Test accuracy: 0.9074.. Top-3 test accuracy: 0.9845\n",
      "Epoch 77/100.. Time per epoch: 139.2485.. Average time per step: 0.3561.. Train loss: 0.0370.. Train accuracy: 0.9873.. Top-3 train accuracy: 0.9997.. Test loss: 0.4646.. Test accuracy: 0.9072.. Top-3 test accuracy: 0.9860\n",
      "Epoch 78/100.. Time per epoch: 139.0921.. Average time per step: 0.3557.. Train loss: 0.0333.. Train accuracy: 0.9885.. Top-3 train accuracy: 0.9998.. Test loss: 0.4639.. Test accuracy: 0.9088.. Top-3 test accuracy: 0.9868\n",
      "Epoch 79/100.. Time per epoch: 139.2366.. Average time per step: 0.3561.. Train loss: 0.0284.. Train accuracy: 0.9903.. Top-3 train accuracy: 0.9998.. Test loss: 0.4368.. Test accuracy: 0.9149.. Top-3 test accuracy: 0.9888\n",
      "Epoch 80/100.. Time per epoch: 139.1471.. Average time per step: 0.3559.. Train loss: 0.0307.. Train accuracy: 0.9894.. Top-3 train accuracy: 0.9998.. Test loss: 0.4596.. Test accuracy: 0.9081.. Top-3 test accuracy: 0.9875\n",
      "Epoch 81/100.. Time per epoch: 139.3094.. Average time per step: 0.3563.. Train loss: 0.0307.. Train accuracy: 0.9889.. Top-3 train accuracy: 0.9998.. Test loss: 0.4486.. Test accuracy: 0.9081.. Top-3 test accuracy: 0.9878\n",
      "Epoch 82/100.. Time per epoch: 139.1995.. Average time per step: 0.3560.. Train loss: 0.0312.. Train accuracy: 0.9891.. Top-3 train accuracy: 0.9998.. Test loss: 0.4845.. Test accuracy: 0.9058.. Top-3 test accuracy: 0.9859\n",
      "Epoch 83/100.. Time per epoch: 139.0099.. Average time per step: 0.3555.. Train loss: 0.0287.. Train accuracy: 0.9902.. Top-3 train accuracy: 0.9998.. Test loss: 0.4959.. Test accuracy: 0.9050.. Top-3 test accuracy: 0.9849\n",
      "Epoch 84/100.. Time per epoch: 139.1334.. Average time per step: 0.3558.. Train loss: 0.0314.. Train accuracy: 0.9886.. Top-3 train accuracy: 0.9999.. Test loss: 0.4603.. Test accuracy: 0.9072.. Top-3 test accuracy: 0.9862\n",
      "Epoch 85/100.. Time per epoch: 139.1492.. Average time per step: 0.3559.. Train loss: 0.0275.. Train accuracy: 0.9903.. Top-3 train accuracy: 0.9999.. Test loss: 0.4652.. Test accuracy: 0.9089.. Top-3 test accuracy: 0.9870\n",
      "Epoch 86/100.. Time per epoch: 139.1452.. Average time per step: 0.3559.. Train loss: 0.0326.. Train accuracy: 0.9884.. Top-3 train accuracy: 0.9998.. Test loss: 0.4395.. Test accuracy: 0.9089.. Top-3 test accuracy: 0.9872\n",
      "Epoch 87/100.. Time per epoch: 139.1707.. Average time per step: 0.3559.. Train loss: 0.0260.. Train accuracy: 0.9911.. Top-3 train accuracy: 0.9998.. Test loss: 0.4733.. Test accuracy: 0.9061.. Top-3 test accuracy: 0.9872\n",
      "Epoch 88/100.. Time per epoch: 139.5164.. Average time per step: 0.3568.. Train loss: 0.0261.. Train accuracy: 0.9907.. Top-3 train accuracy: 0.9998.. Test loss: 0.4604.. Test accuracy: 0.9083.. Top-3 test accuracy: 0.9869\n",
      "Epoch 89/100.. Time per epoch: 139.3151.. Average time per step: 0.3563.. Train loss: 0.0275.. Train accuracy: 0.9904.. Top-3 train accuracy: 0.9999.. Test loss: 0.5190.. Test accuracy: 0.9051.. Top-3 test accuracy: 0.9841\n",
      "Epoch 90/100.. Time per epoch: 139.2178.. Average time per step: 0.3561.. Train loss: 0.0245.. Train accuracy: 0.9912.. Top-3 train accuracy: 0.9999.. Test loss: 0.4926.. Test accuracy: 0.9039.. Top-3 test accuracy: 0.9862\n",
      "Epoch 91/100.. Time per epoch: 139.1599.. Average time per step: 0.3559.. Train loss: 0.0310.. Train accuracy: 0.9893.. Top-3 train accuracy: 0.9998.. Test loss: 0.4732.. Test accuracy: 0.9071.. Top-3 test accuracy: 0.9867\n",
      "Epoch 92/100.. Time per epoch: 139.4510.. Average time per step: 0.3567.. Train loss: 0.0232.. Train accuracy: 0.9922.. Top-3 train accuracy: 0.9999.. Test loss: 0.4436.. Test accuracy: 0.9122.. Top-3 test accuracy: 0.9874\n",
      "Epoch 93/100.. Time per epoch: 139.5022.. Average time per step: 0.3568.. Train loss: 0.0276.. Train accuracy: 0.9901.. Top-3 train accuracy: 0.9998.. Test loss: 0.4681.. Test accuracy: 0.9078.. Top-3 test accuracy: 0.9846\n",
      "Epoch 94/100.. Time per epoch: 139.0729.. Average time per step: 0.3557.. Train loss: 0.0269.. Train accuracy: 0.9908.. Top-3 train accuracy: 0.9998.. Test loss: 0.4800.. Test accuracy: 0.9058.. Top-3 test accuracy: 0.9865\n",
      "Epoch 95/100.. Time per epoch: 138.8984.. Average time per step: 0.3552.. Train loss: 0.0259.. Train accuracy: 0.9906.. Top-3 train accuracy: 0.9998.. Test loss: 0.4814.. Test accuracy: 0.9084.. Top-3 test accuracy: 0.9866\n",
      "Epoch 96/100.. Time per epoch: 138.9953.. Average time per step: 0.3555.. Train loss: 0.0273.. Train accuracy: 0.9903.. Top-3 train accuracy: 0.9999.. Test loss: 0.4773.. Test accuracy: 0.9070.. Top-3 test accuracy: 0.9852\n",
      "Epoch 97/100.. Time per epoch: 139.0952.. Average time per step: 0.3557.. Train loss: 0.0244.. Train accuracy: 0.9914.. Top-3 train accuracy: 0.9999.. Test loss: 0.4580.. Test accuracy: 0.9084.. Top-3 test accuracy: 0.9863\n",
      "Epoch 98/100.. Time per epoch: 139.0053.. Average time per step: 0.3555.. Train loss: 0.0211.. Train accuracy: 0.9927.. Top-3 train accuracy: 0.9998.. Test loss: 0.4837.. Test accuracy: 0.9062.. Top-3 test accuracy: 0.9861\n",
      "Epoch 99/100.. Time per epoch: 138.8969.. Average time per step: 0.3552.. Train loss: 0.0252.. Train accuracy: 0.9913.. Top-3 train accuracy: 0.9999.. Test loss: 0.4867.. Test accuracy: 0.9042.. Top-3 test accuracy: 0.9861\n",
      "Epoch 100/100.. Time per epoch: 139.1126.. Average time per step: 0.3558.. Train loss: 0.0276.. Train accuracy: 0.9905.. Top-3 train accuracy: 0.9999.. Test loss: 0.4750.. Test accuracy: 0.9051.. Top-3 test accuracy: 0.9864\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_DenseNet169_Mish.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
